{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:3: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b6e5b0ffc9c42f6a0bc8fa7b50fd4cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17880.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 一些常规特征\n",
    "import pandas as pd\n",
    "from tqdm.autonotebook import *\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "tqdm.pandas()\n",
    "\n",
    "train = pd.read_csv('../data/train.csv')\n",
    "test = pd.read_csv('../data/test.csv')\n",
    "data = pd.concat([train, test], axis=0, sort=False).reset_index(drop=True)\n",
    "\n",
    "# 将所有列进行拼接\n",
    "data['review'] = data.progress_apply(lambda row:str(row['title']) + ' ' + str(row['location']) + ' ' + str(row['company_profile']) + ' ' + \n",
    "                                   str(row['description']) + ' ' + str(row['department']) + ' ' + str(row['requirements']) + ' ' + str(row['benefits']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unable to import 'smart_open.gcs', disabling that module\n",
      "/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py:743: UserWarning: C extension not loaded, training will be slow. Install a C compiler and reinstall gensim for fast training.\n",
      "  \"C extension not loaded, training will be slow. \"\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3319, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-2-8ab45b5c6cf6>\", line 14, in <module>\n",
      "    w2v = Word2Vec(texts, size=60, seed=1017)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/word2vec.py\", line 783, in __init__\n",
      "    fast_version=FAST_VERSION)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py\", line 763, in __init__\n",
      "    end_alpha=self.min_alpha, compute_loss=compute_loss)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/word2vec.py\", line 910, in train\n",
      "    queue_factor=queue_factor, report_delay=report_delay, compute_loss=compute_loss, callbacks=callbacks)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py\", line 1081, in train\n",
      "    **kwargs)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py\", line 553, in train\n",
      "    total_words=total_words, queue_factor=queue_factor, report_delay=report_delay)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py\", line 489, in _train_epoch\n",
      "    report_delay=report_delay, is_corpus_file_mode=False)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/gensim/models/base_any2vec.py\", line 346, in _log_epoch_progress\n",
      "    report = progress_queue.get()  # blocks if workers too slow\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/queue.py\", line 164, in get\n",
      "    self.not_empty.wait()\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/threading.py\", line 295, in wait\n",
      "    waiter.acquire()\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2034, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 1151, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 319, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 353, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/inspect.py\", line 1490, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/inspect.py\", line 1448, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/inspect.py\", line 742, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/posixpath.py\", line 395, in realpath\n",
      "    path, ok = _joinrealpath(filename[:0], filename, {})\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/posixpath.py\", line 429, in _joinrealpath\n",
      "    if not islink(newpath):\n",
      "  File \"/Users/yphacker/opt/anaconda3/envs/py36/lib/python3.6/posixpath.py\", line 171, in islink\n",
      "    st = os.lstat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from collections import defaultdict\n",
    "text_list = list(data['review'])\n",
    "\n",
    "documents = text_list\n",
    "texts = [[word for word in str(document).split(' ') ] for document in documents]\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text:\n",
    "        frequency[token] += 1\n",
    "texts = [[token for token in text if frequency[token] >= 5] for text in texts]\n",
    "\n",
    "\n",
    "w2v = Word2Vec(texts, size=60, seed=1017)\n",
    "w2v.wv.save_word2vec_format('model/w2v_128.txt')\n",
    "print(\"w2v model done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import gensim\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import numpy as np\n",
    "\n",
    "def get_w2v_avg(word2vec_Path,documents=documents):\n",
    "    texts = documents\n",
    "    w2v_dim = 60\n",
    "    model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_Path, binary=False)\n",
    "    vacab = model.vocab.keys()\n",
    "    w2v_feature = np.zeros((len(texts), w2v_dim))\n",
    "    w2v_feature_avg = np.zeros((len(texts), w2v_dim))\n",
    "\n",
    "    for i, line in enumerate(texts):\n",
    "        num = 0\n",
    "        if line == 'null':\n",
    "            w2v_feature_avg[i, :] = np.zeros(w2v_dim)\n",
    "        else:\n",
    "            for word in line.split():\n",
    "                num += 1\n",
    "                if word==\"\":\n",
    "                    vec =  np.zeros(w2v_dim)\n",
    "                else:\n",
    "                    vec = model[word] if word in vacab else np.zeros(w2v_dim)\n",
    "                w2v_feature[i, :] += vec\n",
    "            w2v_feature_avg[i, :] = w2v_feature[i, :] / num\n",
    "    w2v_avg = pd.DataFrame(w2v_feature_avg)\n",
    "    w2v_avg.columns = ['w2v_avg_' + str(i) for i in w2v_avg.columns]\n",
    "    return w2v_avg\n",
    "w2v_avg_feat=get_w2v_avg('model/w2v_128.txt')\n",
    "print(\"done\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_avg_feat.to_csv('feature/w2v_feature.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
